{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from joblib import Parallel, delayed\n",
    "import time\n",
    "import numpy as np\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "from sklearn.decomposition import PCA,TruncatedSVD\n",
    "from sklearn import manifold\n",
    "import featuretools as ft \n",
    "from itertools import combinations,permutations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_all_data = pd.read_hdf('data/train.h5')\n",
    "test_all_data = pd.read_hdf('data/test.h5')\n",
    "\n",
    "train_all_data[\"type\"] = train_all_data[\"type\"].map({'围网':0,'刺网':1,'拖网':2})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 将面积为0 的点 的速度都设置为0\n",
    "def get_areas(dataframe):\n",
    "    dataframe_ship_group = dataframe.groupby('ship')\n",
    "    x_max_min_Series = dataframe_ship_group.apply(lambda t: t.x.max()-t.x.min())\n",
    "    y_max_min_Series = dataframe_ship_group.apply(lambda t: t.y.max()-t.y.min())\n",
    "    areas = x_max_min_Series.multiply(y_max_min_Series) # 面积\n",
    "    return areas\n",
    "\n",
    "train_areas = get_areas(train_all_data)\n",
    "test_areas = get_areas(test_all_data)\n",
    "\n",
    "zeros_train_areas = train_areas.loc[train_areas==0]\n",
    "zeros_test_areas = test_areas.loc[test_areas==0]\n",
    "\n",
    "train_all_data.loc[zeros_train_areas.index,'v']=0\n",
    "test_all_data.loc[zeros_test_areas.index,'v']=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 获取所有的特征\n",
    "train_all_data[\"ship\"] =[ 'train_{}'.format(i) for i in train_all_data[\"ship\"]]\n",
    "\n",
    "X_train = train_all_data.drop(['type'], axis=1)\n",
    "y_train = train_all_data[\"type\"]\n",
    "\n",
    "test_all_data[\"ship\"] =[ 'test_{}'.format(i) for i in test_all_data[\"ship\"]]\n",
    "X_test = test_all_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2699638, 6)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#组合训练集和测试集，这样省去两次执行相同步骤的麻烦\n",
    "combi = X_train.append(X_test, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#对于数据集，必须具有唯一标识符特征\n",
    "combi[\"id\"] = combi.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Entityset: types\n",
       "  Entities:\n",
       "    ship_entity [Rows: 3482016, Columns: 7]\n",
       "  Relationships:\n",
       "    No relationships"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#我们要创建一个实体集EntitySet。实体集是一种包含多个数据帧及其之间关系的结构\n",
    "# creating and entity set 'es' \n",
    "es = ft.EntitySet(id = 'types') \n",
    "# adding a dataframe \n",
    "es.entity_from_dataframe(entity_id = 'ship_entity', \n",
    "                         dataframe = combi, \n",
    "                         index = 'id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Entityset: types\n",
       "  Entities:\n",
       "    ship_entity [Rows: 3482016, Columns: 2]\n",
       "    one_ship_entity [Rows: 9000, Columns: 6]\n",
       "  Relationships:\n",
       "    ship_entity.ship -> one_ship_entity.ship"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "es.normalize_entity(base_entity_id='ship_entity', \n",
    "                    new_entity_id='one_ship_entity', \n",
    "                    index = 'ship', \n",
    "                    additional_variables =   \n",
    "                    ['x',\"y\",\"v\",\"d\",\"time\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Built 7 features\n",
      "EntitySet scattered to 4 workers in 41 seconds\n",
      "Elapsed: 01:01 | Progress: 100%|██████████\n"
     ]
    }
   ],
   "source": [
    "#现在我们要使用DFS来自动创建新特征。上面提到，DFS使用特征基元和实体集中给出的多个表来创建特征。\n",
    "feature_matrix, feature_names = ft.dfs(entityset=es, \n",
    "                                       target_entity = 'ship_entity', \n",
    "                                       max_depth = 2, \n",
    "                                       verbose = 1, \n",
    "                                       n_jobs = -1)\n",
    "#target_entity只是创建新特征的实体ID，这种情况下为实体“bigmart”。\n",
    "#参数max_depth控制着通过堆叠基元生成的要素复杂性。\n",
    "#参数n_jobs通过使用多个内核来辅助并行特征计算。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['ship', 'one_ship_entity.x', 'one_ship_entity.y', 'one_ship_entity.v',\n",
       "       'one_ship_entity.d', 'one_ship_entity.time',\n",
       "       'one_ship_entity.COUNT(ship_entity)'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#看下这些新创建的特征。\n",
    "feature_matrix.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ship</th>\n",
       "      <th>one_ship_entity.x</th>\n",
       "      <th>one_ship_entity.y</th>\n",
       "      <th>one_ship_entity.v</th>\n",
       "      <th>one_ship_entity.d</th>\n",
       "      <th>one_ship_entity.time</th>\n",
       "      <th>one_ship_entity.COUNT(ship_entity)</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>train_1747</td>\n",
       "      <td>6.109211e+06</td>\n",
       "      <td>5.114873e+06</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1120 23:50:49</td>\n",
       "      <td>421</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>train_1747</td>\n",
       "      <td>6.109211e+06</td>\n",
       "      <td>5.114873e+06</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1120 23:50:49</td>\n",
       "      <td>421</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>train_1747</td>\n",
       "      <td>6.109211e+06</td>\n",
       "      <td>5.114873e+06</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1120 23:50:49</td>\n",
       "      <td>421</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>train_1747</td>\n",
       "      <td>6.109211e+06</td>\n",
       "      <td>5.114873e+06</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1120 23:50:49</td>\n",
       "      <td>421</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>train_1747</td>\n",
       "      <td>6.109211e+06</td>\n",
       "      <td>5.114873e+06</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1120 23:50:49</td>\n",
       "      <td>421</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          ship  one_ship_entity.x  one_ship_entity.y  one_ship_entity.v  \\\n",
       "id                                                                        \n",
       "0   train_1747       6.109211e+06       5.114873e+06                0.0   \n",
       "1   train_1747       6.109211e+06       5.114873e+06                0.0   \n",
       "2   train_1747       6.109211e+06       5.114873e+06                0.0   \n",
       "3   train_1747       6.109211e+06       5.114873e+06                0.0   \n",
       "4   train_1747       6.109211e+06       5.114873e+06                0.0   \n",
       "\n",
       "    one_ship_entity.d one_ship_entity.time  one_ship_entity.COUNT(ship_entity)  \n",
       "id                                                                              \n",
       "0                   0        1120 23:50:49                                 421  \n",
       "1                   0        1120 23:50:49                                 421  \n",
       "2                   0        1120 23:50:49                                 421  \n",
       "3                   0        1120 23:50:49                                 421  \n",
       "4                   0        1120 23:50:49                                 421  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_matrix.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#这个数据帧存在一个问题，即未正确排序。我们必须根据combi数据帧中的id变量对其进行排序。\n",
    "feature_matrix = feature_matrix.reindex(index=combi['id']) \n",
    "feature_matrix = feature_matrix.reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "combi_feature_matrix = pd.merge(combi,feature_matrix, on='id',how='left')\n",
    "\n",
    "combi_feature_matrix.drop(['id'], axis=1, inplace=True) \n",
    "\n",
    "train = combi_feature_matrix[:2699638] \n",
    "test = combi_feature_matrix[2699638:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "train[\"ship\"] =[ i.replace('train_', '') for i in train[\"ship_x\"]]\n",
    "train.drop(['ship_x','ship_y'], axis=1, inplace=True) \n",
    "\n",
    "test[\"ship\"] =[ i.replace('test_', '') for i in test[\"ship_x\"]]\n",
    "test.drop(['ship_x','ship_y'], axis=1, inplace=True) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
